import pandas as pd
import time
import xgboost as xgb
from sklearn.model_selection import train_test_split

def read_data(path):
    train_data=pd.read_csv(path+'train.csv',dtype={'sale_date':object,'is_pro':object,},encoding='utf-8')
    train_data['goodsid']=train_data['goodsid'].astype(int)
    train_data['goodsid']=train_data['goodsid'].astype('object')#导入训练集train.csv

    test_data=pd.read_csv(path+'test.csv',dtype={'sale_date':object,'is_pro':object},encoding='utf-8')
    test_data['goodsid']=test_data['goodsid'].astype(int)
    test_data['goodsid']=test_data['goodsid'].astype('object')#导入测试集test_data.csv

    date_ch=pd.read_csv(path+'date_ch.csv',dtype={'dim_date_id':object,'year_code':object,'day_week_cn':object,'week_day_code':object,'is_weekend':object,'official_holiday_code':object},encoding='utf-8')
    date_ch.rename(columns={'dim_date_id':'sale_date'}, inplace=True)#导入日期明细表date_ch.csv

    goods_ch=pd.read_csv(path+'goods_ch.csv',dtype={'catg_l_id':object,'season_class':object},encoding='utf-8')
    goods_ch['goodsid']=goods_ch['goodsid'].astype(int)
    goods_ch['goodsid']=goods_ch['goodsid'].astype('object')#导入商品明细表goods_ch.csv
    del date_ch['official_holiday_name']
    del date_ch['festival_name']
    del goods_ch['season_class_name']
    del goods_ch['div_name']#删除部分数据列，因为它们可用其他列代替
    return train_data,test_data,date_ch,goods_ch

def merge_attr(train_data,test_data,date_ch,goods_ch):
    raw_train=pd.merge(train_data,date_ch,how='left',on='sale_date')
    raw_train=pd.merge(raw_train,goods_ch,how='left',on='goodsid')#将商品明细、日期明细合并到训练集中
    raw_test=pd.merge(test_data,date_ch,how='left',on='sale_date')
    raw_test=pd.merge(raw_test,goods_ch,how='left',on='goodsid')#将商品明细、日期明细合并到测试集中
    return raw_train,raw_test

def preprocessing(path):
    train_data,test_data,date_ch,goods_ch=read_data(path)#读入数据
    raw_train,raw_test=merge_attr(train_data,test_data,date_ch,goods_ch)#合并数据
    n_train=raw_train.shape[0]
    train_cols=list(range(3,6))+[7]+list(range(11,14))+[15]+[17]+[20]#取出准备使用的数据列，其他数据丢弃
    test_cols=list(range(2,5))+[6]+list(range(10,13))+[14]+[16]+[19]#取出准备使用的数据列，其他数据丢弃
    all_features=pd.concat((raw_train.iloc[:,train_cols], raw_test.iloc[:,test_cols]))#先将训练集与测试集合并，统一进行标准化
    labels=raw_train.iloc[:,[2]][:n_train]#取出训练集中的销量数据，避免被一并标准化
    info=raw_test.iloc[:,[0,1]]#保存测试集中的商品id、日期id数据，以免丢失

    numeric_features=all_features.dtypes[all_features.dtypes!='object'].index
    all_features[numeric_features]=all_features[numeric_features].apply(lambda x:(x-x.mean())/(x.std()))
    all_features[numeric_features]=all_features[numeric_features].fillna(0)#连续数据标准化
    
    attr_list=['year_code', 'day_week_cn','is_weekend','festival_code', 'season_class','catg_l_id','is_pro','week_day_code']#需要扩增的离散型数据列
    all_features=pd.get_dummies(all_features,dummy_na=True,prefix=attr_list,columns=attr_list,dtype=int)#离散数据标准化
    
    train_features=all_features[:n_train]
    test_features=all_features[n_train:]#再拆分为训练集、测试集，这里的数据均已经过标准化
    train_features=pd.concat([train_features,labels],axis=1)#将销量数据加回训练集中
    
    return train_features,test_features,info

def wmape(preds, gtrain):
    '''
    自定义评价函数，采用加权平均百分比误差系数，用于xgboost模型
    '''
    label=gtrain.get_label()
    wmape=sum(abs(preds-label))/sum(label)
    return 'wmape',wmape

def xgboost_test(train_features,test_features):
    '''
    本函数仅用于调参，所得预测不可作为提交结果
    （将训练集分为训练集和验证集，以验证集为参考，输出模型每轮训练效果）
    '''
    X=train_features.iloc[:,:-1]
    Y=train_features.iloc[:,[-1]]
    x_train,x_test,y_train,y_test=train_test_split(X,Y,test_size=0.2,random_state=3)
    test=xgb.DMatrix(test_features)
    gtrain=xgb.DMatrix(x_train,label=y_train)
    gtest=xgb.DMatrix(x_test,label=y_test)
    params={'booster':'gbtree',#可选：线性gblinear、树dart
       'eta':0.02,#学习率，0-1之间，增大该值使算法更激进
       'max_depth':10,#树的最大深度，增大该值使算法更复杂，但易过拟合
       'subsample':0.99,#子样本比率，增大该值可以预防过拟合
       'colsample_bytree':0.85,#构建每棵树时列的子样本比率
       }
    num_round=60
    watchlist = [(gtest, "eval"),(gtrain, "train")]
    model=xgb.train(params,gtrain,num_round,watchlist,feval=wmape)
    pred=model.predict(test)
    return pred

def xgboost(train_features,test_features):
    '''
    在xgboost_test函数中调试好参数后，将参数代到这里，得到最终预测结果
    （训练集中的数据全部用于训练模型）
    '''
    X=train_features.iloc[:,:-1]
    Y=train_features.iloc[:,[-1]]
    test=xgb.DMatrix(test_features)
    gtrain=xgb.DMatrix(X,label=Y)
    params={'booster':'gbtree',#可选：线性gblinear、树dart
       'eta':0.02,#学习率，0-1之间，增大该值使算法更激进
       'max_depth':10,#树的最大深度，增大该值使算法更复杂，但易过拟合
       'subsample':0.99,#子样本比率，增大该值可以预防过拟合
       'colsample_bytree':0.85,#构建每棵树时列的子样本比率
       }
    num_round=60#0.655
    model=xgb.train(params,gtrain,num_round,feval=wmape)
    pred=model.predict(test)
    return pred

def result(pred,info):
    res=pd.DataFrame(data=info)
    res['sale_qty']=pred
    res.to_csv('D:\学习\大三下\人工智能与深度学习\deep-learing-project\\xgboost_result.csv')

def main():
    path="D:\学习\大三下\人工智能与深度学习\小组作业\期末作业\预测组赛题\\"
    begin=time.time()
    train_features,test_features,info=preprocessing(path)
    print("数据预处理完成")
    pred=xgboost(train_features,test_features)
    #pred=xgboost_test(train_features,test_features)
    result(pred,info)
    finish=time.time()
    print("用时:{}".format(finish-begin))
    
if __name__=="__main__":
    main()